Setup
workflow_name <- "netnav_03_parameter_recovery"
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr 1.1.4 ✔ readr 2.1.5
## ✔ forcats 1.0.0 ✔ stringr 1.5.1
## ✔ ggplot2 3.4.4 ✔ tibble 3.2.1
## ✔ lubridate 1.9.3 ✔ tidyr 1.3.1
## ✔ purrr 1.0.2
## ── Conflicts ────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(here)
## here() starts at /Users/jaeyoungson/Documents/GitHub/network-navigation-replay
library(broom)
library(patchwork)
source(here("code", "utils", "ggplot_themes.R"))
source(here("code", "utils", "kable_utils.R"))
##
## Attaching package: 'kableExtra'
##
## The following object is masked from 'package:dplyr':
##
## group_rows
knitting <- knitr::is_html_output()
create_path <- function(this_path) {
if (!dir.exists(this_path)) {
dir.create(this_path, recursive = TRUE)
}
}
if (knitting) {
here("outputs", workflow_name) %>%
create_path()
here("figures") %>%
create_path()
}
Models with a lapse rate
In the previous iteration of this work, we had estimated models
containing lapse rate parameters. The rationale was that there might be
some amount of irreducible decision noise in subjects’ behaviors (e.g.,
occasional lapses in attention, resulting in random responses). However,
it’s entirely possible that the lapse rate cannot be reliably estimated
given our data, and if this is true, trying to estimate a lapse rate
could end up introducing other kinds of biases in our parameter
estimation.
lapse_params_true <- here("data", "simulated_model_behaviors") %>%
fs::dir_ls(regexp = "with_lapse\\.csv") %>%
map_dfr(.f = ~read_csv(.x, show_col_types = FALSE), .id = "filename") %>%
mutate(
model = str_extract(filename, "bfs_(backward|forward)|ideal_obs|sr")
) %>%
select(
model, sub_id, lapse_rate, search_threshold, softmax_temperature, sr_gamma
) %>%
distinct() %>%
pivot_longer(
-c(model, sub_id), names_to = "param_name", values_to = "param_value"
) %>%
drop_na()
SR with lapse rate
Starting with the SR, we can see that recovery of the SR gamma
parameter is pretty dismal.
lapse_params_est_sr <- here("data", "param_recovery", "sr_with_lapse") %>%
fs::dir_ls(glob = "*.csv") %>%
map_dfr(.f = ~read_csv(.x, show_col_types = FALSE), .id = "filename") %>%
mutate(
sub_id = str_extract(filename, "sub_[[:digit:]]+"),
sub_id = str_remove(sub_id, "sub_"),
sub_id = as.numeric(sub_id)
) %>%
select(sub_id, everything(), -filename) %>%
rename(neg_loglik = optim_value) %>%
# Find best-fitting optimization run
filter(convergence == "converged") %>%
group_by(sub_id) %>%
slice_min(neg_loglik, n = 1) %>%
ungroup() %>%
# Some subjects may have had multiple "best" optimization runs
# In that case, just go with whichever "best" run was estimated first
group_by(sub_id) %>%
slice_min(optimizer_run, n = 1) %>%
ungroup()
recovery_lapse_sr <- lapse_params_true %>%
filter(model == "sr") %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "true_"
) %>%
left_join(
lapse_params_est_sr %>%
select(sub_id, param_name, param_value = param_value_human_readable) %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "est_"
),
by = join_by(sub_id)
)
plot_recovery_sr_lapse_gamma <- recovery_lapse_sr %>%
ggplot(aes(x=true_sr_gamma, y=est_sr_gamma)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True SR gamma") +
ylab("Estimated SR gamma") +
ggtitle("Parameter recovery: SR w/ lapse rate")
plot_recovery_sr_lapse_gamma

recovery_lapse_sr %>%
with(cor.test(true_sr_gamma, est_sr_gamma, method = "spearman")) %>%
tidy() %>%
kable_custom("Parameter recovery: SR w/ lapse rate")
## Warning in cor.test.default(true_sr_gamma, est_sr_gamma, method = "spearman"):
## Cannot compute exact p-value with ties
Parameter recovery: SR w/ lapse rate
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.36
|
13326088
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
The recovery of the softmax temperature parameter is also fairly bad.
We note that some of the softmax temperatures are estimated to be, on an
absolute scale, very large.
recovery_lapse_sr %>%
ggplot(aes(x=true_softmax_temperature, y=est_softmax_temperature)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True softmax temperature") +
ylab("Estimated softmax temperature") +
ggtitle("Parameter recovery: SR w/ lapse rate")

recovery_lapse_sr %>%
ggplot(aes(x=true_softmax_temperature, y=est_softmax_temperature)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True softmax temperature") +
ylab("Estimated softmax temperature") +
coord_cartesian(ylim = c(-50000, 50000)) +
ggtitle("Parameter recovery: SR w/ lapse rate")

plot_recovery_sr_lapse_softmax <- recovery_lapse_sr %>%
ggplot(aes(x=true_softmax_temperature, y=est_softmax_temperature)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True softmax temperature") +
ylab("Estimated softmax temperature") +
coord_cartesian(ylim = c(-5000, 5000)) +
ggtitle("Parameter recovery: SR w/ lapse rate")
plot_recovery_sr_lapse_softmax

recovery_lapse_sr %>%
with(
cor.test(
true_softmax_temperature, est_softmax_temperature,
method = "spearman"
)
) %>%
tidy() %>%
kable_custom("Parameter recovery: SR w/ lapse rate")
Parameter recovery: SR w/ lapse rate
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.066
|
19449856
|
0.138
|
Spearman’s rank correlation rho
|
two.sided
|
And to round everything out, we can see that we’re unable to reliably
recover the lapse rate.
plot_recovery_sr_lapse_lapse <- recovery_lapse_sr %>%
ggplot(aes(x=true_lapse_rate, y=est_lapse_rate)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True lapse rate") +
ylab("Estimated lapse rate") +
ggtitle("Parameter recovery: SR w/ lapse rate")
plot_recovery_sr_lapse_lapse

recovery_lapse_sr %>%
with(cor.test(true_lapse_rate, est_lapse_rate, method = "spearman")) %>%
tidy() %>%
kable_custom("Parameter recovery: SR w/ lapse rate")
## Warning in cor.test.default(true_lapse_rate, est_lapse_rate, method =
## "spearman"): Cannot compute exact p-value with ties
Parameter recovery: SR w/ lapse rate
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.29
|
14801170
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
Let’s create a visualization combining these plots:
plot_recovery_sr_lapse_combined <- wrap_plots(
plot_recovery_sr_lapse_gamma + ggtitle("Gamma"),
plot_recovery_sr_lapse_softmax + ggtitle("Softmax temperature"),
plot_recovery_sr_lapse_lapse + ggtitle("Lapse rate")
) +
plot_annotation(
title = "Parameter recovery: Successor Representation with lapse rate",
theme = theme(plot.title = element_text(hjust = 0.5)),
tag_levels = "A",
tag_suffix = "."
) &
xlab("True parameter value") &
ylab("Estimated parameter value")
plot_recovery_sr_lapse_combined

if (knitting) {
ggsave(
filename = here("outputs", workflow_name, "param_recovery_sr_lapse.pdf"),
plot = plot_recovery_sr_lapse_combined,
width = 8, height = 4,
units = "in", dpi = 300
)
}
BFS-forward
We had previously also estimated a model of (forward)-BFS with a
lapse rate. Here, we’ll test the recoverability of this model.
Results indicate recovery of the search threshold parameter is
mediocre.
lapse_params_est_bfs_forward <- here(
"data", "param_recovery", "bfs_forward_with_lapse"
) %>%
fs::dir_ls(glob = "*.csv") %>%
map_dfr(.f = ~read_csv(.x, show_col_types = FALSE), .id = "filename") %>%
mutate(
sub_id = str_extract(filename, "sub_[[:digit:]]+"),
sub_id = str_remove(sub_id, "sub_"),
sub_id = as.numeric(sub_id)
) %>%
select(sub_id, everything(), -filename) %>%
rename(neg_loglik = optim_value) %>%
# Find best-fitting optimization run
filter(convergence == "converged") %>%
group_by(sub_id) %>%
slice_min(neg_loglik, n = 1) %>%
ungroup() %>%
# Some subjects may have had multiple "best" optimization runs
# In that case, just go with whichever "best" run was estimated first
group_by(sub_id) %>%
slice_min(optimizer_run, n = 1) %>%
ungroup()
recovery_lapse_bfs_forward <- lapse_params_true %>%
filter(model == "bfs_forward") %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "true_"
) %>%
left_join(
lapse_params_est_bfs_forward %>%
select(sub_id, param_name, param_value = param_value_human_readable) %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "est_"
),
by = join_by(sub_id)
)
plot_recovery_bfs_forward_lapse_threshold <- recovery_lapse_bfs_forward %>%
ggplot(aes(x=true_search_threshold, y=est_search_threshold)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True search threshold") +
ylab("Estimated search threshold") +
ggtitle("Parameter recovery: BFS-forward w/ lapse rate")
plot_recovery_bfs_forward_lapse_threshold

recovery_lapse_bfs_forward %>%
with(
cor.test(true_search_threshold, est_search_threshold, method = "spearman")
) %>%
tidy() %>%
kable_custom("Parameter recovery: BFS-forward w/ lapse rate")
## Warning in cor.test.default(true_search_threshold, est_search_threshold, :
## Cannot compute exact p-value with ties
Parameter recovery: BFS-forward w/ lapse rate
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.547
|
9439157
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
The recovery of the lapse rate is also mediocre.
plot_recovery_bfs_forward_lapse_lapse <- recovery_lapse_bfs_forward %>%
ggplot(aes(x=true_lapse_rate, y=est_lapse_rate)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True lapse") +
ylab("Estimated lapse rate") +
ggtitle("Parameter recovery: BFS-forward w/ lapse rate")
plot_recovery_bfs_forward_lapse_lapse

recovery_lapse_bfs_forward %>%
with(cor.test(true_lapse_rate, est_lapse_rate, method = "spearman")) %>%
tidy() %>%
kable_custom("Parameter recovery: BFS-forward w/ lapse rate")
## Warning in cor.test.default(true_lapse_rate, est_lapse_rate, method =
## "spearman"): Cannot compute exact p-value with ties
Parameter recovery: BFS-forward w/ lapse rate
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.536
|
9659045
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
Plotting them together…
plot_recovery_bfs_forward_lapse_combined <- wrap_plots(
plot_recovery_bfs_forward_lapse_threshold + ggtitle("Search threshold"),
plot_recovery_bfs_forward_lapse_lapse + ggtitle("Lapse rate")
) +
plot_annotation(
title = "Parameter recovery: BFS-forward",
theme = theme(plot.title = element_text(hjust = 0.5)),
tag_levels = "A",
tag_suffix = "."
) &
xlab("True parameter value") &
ylab("Estimated parameter value")
plot_recovery_bfs_forward_lapse_combined

if (knitting) {
ggsave(
filename = here(
"outputs", workflow_name, "param_recovery_bfs_forward_lapse.pdf"
),
plot = plot_recovery_bfs_forward_lapse_combined,
width = 8, height = 4,
units = "in", dpi = 300
)
}
Models without a lapse rate
Since it appears that lapse rates cannot be reliably recovered, we’ll
now try looking at models that don’t contain a lapse rate. Note that
these include new models that we’ve added in the revision.
params_true <- here("data", "simulated_model_behaviors") %>%
fs::dir_ls(regexp = "no_lapse\\.csv") %>%
map_dfr(.f = ~read_csv(.x, show_col_types = FALSE), .id = "filename") %>%
mutate(
model = str_extract(filename, "bfs_(backward|forward)|ideal_obs|sr")
) %>%
select(model, sub_id, search_threshold, softmax_temperature, sr_gamma) %>%
distinct() %>%
pivot_longer(
-c(model, sub_id), names_to = "param_name", values_to = "param_value"
) %>%
drop_na()
params_est <- here("data", "param_recovery") %>%
fs::dir_ls(
regexp = "(bfs_(backward|forward)|ideal_obs|sr)_no_lapse(.)+\\.csv",
recurse = 1
) %>%
map_dfr(.f = ~read_csv(.x, show_col_types = FALSE), .id = "filename") %>%
mutate(
sub_id = str_extract(filename, "sub_[[:digit:]]+"),
sub_id = str_remove(sub_id, "sub_"),
sub_id = as.numeric(sub_id),
model = str_extract(filename, "bfs_(backward|forward)|ideal_obs|sr")
) %>%
select(sub_id, everything(), -filename) %>%
rename(neg_loglik = optim_value) %>%
# Find best-fitting optimization run
filter(convergence == "converged") %>%
group_by(sub_id, model) %>%
slice_min(neg_loglik, n = 1) %>%
ungroup() %>%
# Some subjects may have had multiple "best" optimization runs
# In that case, just go with whichever "best" run was estimated first
group_by(sub_id, model) %>%
slice_min(optimizer_run, n = 1) %>%
ungroup() %>%
# Clean up
mutate(
param_value_to_keep = if_else(
is.na(param_value), param_value_human_readable, param_value
)
) %>%
select(
model, sub_id, param_name, param_value = param_value_to_keep, neg_loglik
) %>%
arrange(model, sub_id, param_name)
BFS-backward
Recovery is generally very good. We note that the optimizer tends to
converge upon two (inaccurate) modes when the true search threshold is
greater than 10. We know that behavior basically asymptotes around then,
so estimated thresholds greater than 10 should probably be treated as
being indistinguishable from asymptotic.
recovery_bfs_backward <- params_true %>%
filter(model == "bfs_backward") %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "true_"
) %>%
left_join(
params_est %>%
filter(model == "bfs_backward") %>%
select(sub_id, param_name, param_value) %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "est_"
),
by = join_by(sub_id)
)
plot_recovery_bfs_backward <- recovery_bfs_backward %>%
ggplot(aes(x=true_search_threshold, y=est_search_threshold)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True search threshold") +
ylab("Estimated search threshold") +
ggtitle("Parameter recovery: BFS-backward")
plot_recovery_bfs_backward

recovery_bfs_backward %>%
with(
cor.test(true_search_threshold, est_search_threshold, method = "spearman")
) %>%
tidy() %>%
kable_custom("Parameter recovery: BFS-backward")
## Warning in cor.test.default(true_search_threshold, est_search_threshold, :
## Cannot compute exact p-value with ties
Parameter recovery: BFS-backward
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.975
|
530753.1
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
BFS-forward
Recovery is similarly good for the BFS-forward model, though we note
that the optimizer sometimes overestimates large thresholds (starting at
around 12) and underestimates small thresholds (ending at around 4).
recovery_bfs_forward <- params_true %>%
filter(model == "bfs_forward") %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "true_"
) %>%
left_join(
params_est %>%
filter(model == "bfs_forward") %>%
select(sub_id, param_name, param_value) %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "est_"
),
by = join_by(sub_id)
)
plot_recovery_bfs_forward <- recovery_bfs_forward %>%
ggplot(aes(x=true_search_threshold, y=est_search_threshold)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True search threshold") +
ylab("Estimated search threshold") +
ggtitle("Parameter recovery: BFS-forward")
plot_recovery_bfs_forward

recovery_bfs_forward %>%
with(
cor.test(true_search_threshold, est_search_threshold, method = "spearman")
) %>%
tidy() %>%
kable_custom("Parameter recovery: BFS-forward")
Parameter recovery: BFS-forward
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.971
|
595902
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
Ideal observer
Parameter recovery for the ideal observer model is clearly biased,
such that the optimizer is best at recovering (inverse) temperatures in
the range of [-1, 0], less good at recovering temperatures in the range
of [-2, -1], and much worse beyond that. It may be wise to interpret
estimated temperatures beyond -2 as being “very consistent” with an
ideal observer model.
recovery_ideal_obs <- params_true %>%
filter(model == "ideal_obs") %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "true_"
) %>%
left_join(
params_est %>%
filter(model == "ideal_obs") %>%
select(sub_id, param_name, param_value) %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "est_"
),
by = join_by(sub_id)
)
plot_recovery_ideal_obs <- recovery_ideal_obs %>%
ggplot(aes(x=true_softmax_temperature, y=est_softmax_temperature)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True softmax temperature") +
ylab("Estimated softmax temperature") +
ggtitle("Parameter recovery: Ideal observer")
plot_recovery_ideal_obs

recovery_ideal_obs %>%
with(
cor.test(
true_softmax_temperature, est_softmax_temperature,
method = "spearman"
)
) %>%
tidy() %>%
kable_custom("Parameter recovery: Ideal observer")
## Warning in cor.test.default(true_softmax_temperature, est_softmax_temperature,
## : Cannot compute exact p-value with ties
Parameter recovery: Ideal observer
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.909
|
1905067
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
Successor Representation
We find that the SR gamma parameter has acceptable recoverability,
though we also note that smaller values of gamma appear to be a little
less reliably recovered: true gammas below 0.25 can be underestimated as
near-zero.
recovery_sr <- params_true %>%
filter(model == "sr") %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "true_"
) %>%
left_join(
params_est %>%
filter(model == "sr") %>%
select(sub_id, param_name, param_value) %>%
pivot_wider(
names_from = param_name,
values_from = param_value,
names_prefix = "est_"
),
by = join_by(sub_id)
)
plot_recovery_sr_gamma <- recovery_sr %>%
ggplot(aes(x=true_sr_gamma, y=est_sr_gamma)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True gamma") +
ylab("Estimated gamma") +
ggtitle("Parameter recovery: Successor Rep.")
plot_recovery_sr_gamma

recovery_sr %>%
with(cor.test(true_sr_gamma, est_sr_gamma, method = "spearman")) %>%
tidy() %>%
kable_custom("Parameter recovery: Successor Rep.")
## Warning in cor.test.default(true_sr_gamma, est_sr_gamma, method = "spearman"):
## Cannot compute exact p-value with ties
Parameter recovery: Successor Rep.
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.807
|
4023600
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
Recovery of the softmax temperature parameter is less good. We can
see that the optimizer, on occasion, estimates extremely large values of
this parameter. Below, we plot all of the data in the first graph, then
zoom in to see more typical values in the second graph. The third graph
zooms in a bit more to emphasize that past true softmax temperatures of
about 250, the optimizer often finds a solution at around 4000.
Thankfully, the softmax temperature is not a parameter we’re deeply
interested in interpreting, so as long as it doesn’t interfere with
recovery of gamma (which it appears not to), it suffices that large
(inverse) temperatures reflect “strong weighting.”
recovery_sr %>%
ggplot(aes(x=true_softmax_temperature, y=est_softmax_temperature)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True softmax temperature") +
ylab("Estimated softmax temperature") +
ggtitle("Parameter recovery: Successor Rep.")

recovery_sr %>%
ggplot(aes(x=true_softmax_temperature, y=est_softmax_temperature)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True softmax temperature") +
ylab("Estimated softmax temperature") +
ggtitle("Parameter recovery: Successor Rep.") +
coord_cartesian(ylim = c(-100, 20000))

plot_recovery_sr_softmax <- recovery_sr %>%
ggplot(aes(x=true_softmax_temperature, y=est_softmax_temperature)) +
theme_custom() +
geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "blue") +
geom_point(alpha = 0.25) +
xlab("True softmax temperature") +
ylab("Estimated softmax temperature") +
ggtitle("Parameter recovery: Successor Rep.") +
coord_cartesian(ylim = c(-100, 5000))
plot_recovery_sr_softmax

recovery_sr %>%
with(
cor.test(
true_softmax_temperature, est_softmax_temperature,
method = "spearman"
)
) %>%
tidy() %>%
kable_custom("Parameter recovery: Successor Rep.")
Parameter recovery: Successor Rep.
|
estimate
|
statistic
|
p.value
|
method
|
alternative
|
|
0.53
|
9789122
|
0
|
Spearman’s rank correlation rho
|
two.sided
|
Plot for supplement
We’ll now combine all of these parameter recovery plots…
plot_recovery_combined <- (
(plot_recovery_bfs_backward + ggtitle("BFS-backward")) +
(plot_recovery_bfs_forward + ggtitle("BFS-forward")) +
(plot_recovery_ideal_obs + ggtitle("Ideal observer"))
) /
(
(plot_recovery_sr_gamma + ggtitle("Successor Representation gamma")) +
(plot_recovery_sr_softmax + ggtitle("Successor Representation softmax"))
) +
plot_annotation(
title = "Parameter recovery",
theme = theme(plot.title = element_text(hjust = 0.5)),
tag_levels = "A",
tag_suffix = "."
)
plot_recovery_combined

if (knitting) {
ggsave(
filename = here("outputs", workflow_name, "param_recovery_combined.pdf"),
plot = plot_recovery_combined,
width = 8, height = 6,
units = "in", dpi = 300
)
# Redundant copy
ggsave(
filename = here("figures", "supp_param_recovery.pdf"),
plot = plot_recovery_combined,
width = 8, height = 6,
units = "in", dpi = 300
)
}
LS0tCnRpdGxlOiAiUGFyYW1ldGVyIHJlY292ZXJ5IgpvdXRwdXQ6CiAgaHRtbF9kb2N1bWVudDoKICAgIGNvZGVfZG93bmxvYWQ6IHRydWUKICAgIGNvZGVfZm9sZGluZzogaGlkZQogICAgdG9jOiB0cnVlCiAgICB0b2NfZmxvYXQ6CiAgICAgIGNvbGxhcHNlZDogdHJ1ZQotLS0KCiMgU2V0dXAKCmBgYHtyIGxpYnJhcmllc30Kd29ya2Zsb3dfbmFtZSA8LSAibmV0bmF2XzAzX3BhcmFtZXRlcl9yZWNvdmVyeSIKCmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KGhlcmUpCmxpYnJhcnkoYnJvb20pCmxpYnJhcnkocGF0Y2h3b3JrKQoKc291cmNlKGhlcmUoImNvZGUiLCAidXRpbHMiLCAiZ2dwbG90X3RoZW1lcy5SIikpCnNvdXJjZShoZXJlKCJjb2RlIiwgInV0aWxzIiwgImthYmxlX3V0aWxzLlIiKSkKCmtuaXR0aW5nIDwtIGtuaXRyOjppc19odG1sX291dHB1dCgpCgpjcmVhdGVfcGF0aCA8LSBmdW5jdGlvbih0aGlzX3BhdGgpIHsKICBpZiAoIWRpci5leGlzdHModGhpc19wYXRoKSkgewogICAgZGlyLmNyZWF0ZSh0aGlzX3BhdGgsIHJlY3Vyc2l2ZSA9IFRSVUUpCiAgfQp9CgppZiAoa25pdHRpbmcpIHsKICBoZXJlKCJvdXRwdXRzIiwgd29ya2Zsb3dfbmFtZSkgJT4lCiAgICBjcmVhdGVfcGF0aCgpCiAgCiAgaGVyZSgiZmlndXJlcyIpICU+JQogICAgY3JlYXRlX3BhdGgoKQp9CmBgYAoKCiMgTW9kZWxzIHdpdGggYSBsYXBzZSByYXRlCgpJbiB0aGUgcHJldmlvdXMgaXRlcmF0aW9uIG9mIHRoaXMgd29yaywgd2UgaGFkIGVzdGltYXRlZCBtb2RlbHMgY29udGFpbmluZyBsYXBzZSByYXRlIHBhcmFtZXRlcnMuIFRoZSByYXRpb25hbGUgd2FzIHRoYXQgdGhlcmUgbWlnaHQgYmUgc29tZSBhbW91bnQgb2YgaXJyZWR1Y2libGUgZGVjaXNpb24gbm9pc2UgaW4gc3ViamVjdHMnIGJlaGF2aW9ycyAoZS5nLiwgb2NjYXNpb25hbCBsYXBzZXMgaW4gYXR0ZW50aW9uLCByZXN1bHRpbmcgaW4gcmFuZG9tIHJlc3BvbnNlcykuIEhvd2V2ZXIsIGl0J3MgZW50aXJlbHkgcG9zc2libGUgdGhhdCB0aGUgbGFwc2UgcmF0ZSBjYW5ub3QgYmUgcmVsaWFibHkgZXN0aW1hdGVkIGdpdmVuIG91ciBkYXRhLCBhbmQgaWYgdGhpcyBpcyB0cnVlLCB0cnlpbmcgdG8gZXN0aW1hdGUgYSBsYXBzZSByYXRlIGNvdWxkIGVuZCB1cCBpbnRyb2R1Y2luZyBvdGhlciBraW5kcyBvZiBiaWFzZXMgaW4gb3VyIHBhcmFtZXRlciBlc3RpbWF0aW9uLgoKYGBge3IgbG9hZC1sYXBzZS1wYXJhbXMtdHJ1ZX0KbGFwc2VfcGFyYW1zX3RydWUgPC0gaGVyZSgiZGF0YSIsICJzaW11bGF0ZWRfbW9kZWxfYmVoYXZpb3JzIikgJT4lCiAgZnM6OmRpcl9scyhyZWdleHAgPSAid2l0aF9sYXBzZVxcLmNzdiIpICU+JQogIG1hcF9kZnIoLmYgPSB+cmVhZF9jc3YoLngsIHNob3dfY29sX3R5cGVzID0gRkFMU0UpLCAuaWQgPSAiZmlsZW5hbWUiKSAlPiUKICBtdXRhdGUoCiAgICBtb2RlbCA9IHN0cl9leHRyYWN0KGZpbGVuYW1lLCAiYmZzXyhiYWNrd2FyZHxmb3J3YXJkKXxpZGVhbF9vYnN8c3IiKQogICkgJT4lCiAgc2VsZWN0KAogICAgbW9kZWwsIHN1Yl9pZCwgbGFwc2VfcmF0ZSwgc2VhcmNoX3RocmVzaG9sZCwgc29mdG1heF90ZW1wZXJhdHVyZSwgc3JfZ2FtbWEKICApICU+JQogIGRpc3RpbmN0KCkgJT4lCiAgcGl2b3RfbG9uZ2VyKAogICAgLWMobW9kZWwsIHN1Yl9pZCksIG5hbWVzX3RvID0gInBhcmFtX25hbWUiLCB2YWx1ZXNfdG8gPSAicGFyYW1fdmFsdWUiCiAgKSAlPiUKICBkcm9wX25hKCkKYGBgCgojIyMgU1Igd2l0aCBsYXBzZSByYXRlCgpTdGFydGluZyB3aXRoIHRoZSBTUiwgd2UgY2FuIHNlZSB0aGF0IHJlY292ZXJ5IG9mIHRoZSBTUiBnYW1tYSBwYXJhbWV0ZXIgaXMgcHJldHR5IGRpc21hbC4KCmBgYHtyIHJlY292ZXJ5LWxhcHNlLXNyLWdhbW1hfQpsYXBzZV9wYXJhbXNfZXN0X3NyIDwtIGhlcmUoImRhdGEiLCAicGFyYW1fcmVjb3ZlcnkiLCAic3Jfd2l0aF9sYXBzZSIpICU+JQogIGZzOjpkaXJfbHMoZ2xvYiA9ICIqLmNzdiIpICU+JQogIG1hcF9kZnIoLmYgPSB+cmVhZF9jc3YoLngsIHNob3dfY29sX3R5cGVzID0gRkFMU0UpLCAuaWQgPSAiZmlsZW5hbWUiKSAlPiUKICBtdXRhdGUoCiAgICBzdWJfaWQgPSBzdHJfZXh0cmFjdChmaWxlbmFtZSwgInN1Yl9bWzpkaWdpdDpdXSsiKSwKICAgIHN1Yl9pZCA9IHN0cl9yZW1vdmUoc3ViX2lkLCAic3ViXyIpLAogICAgc3ViX2lkID0gYXMubnVtZXJpYyhzdWJfaWQpCiAgKSAlPiUKICBzZWxlY3Qoc3ViX2lkLCBldmVyeXRoaW5nKCksIC1maWxlbmFtZSkgJT4lCiAgcmVuYW1lKG5lZ19sb2dsaWsgPSBvcHRpbV92YWx1ZSkgJT4lCiAgIyBGaW5kIGJlc3QtZml0dGluZyBvcHRpbWl6YXRpb24gcnVuCiAgZmlsdGVyKGNvbnZlcmdlbmNlID09ICJjb252ZXJnZWQiKSAlPiUKICBncm91cF9ieShzdWJfaWQpICU+JQogIHNsaWNlX21pbihuZWdfbG9nbGlrLCBuID0gMSkgJT4lCiAgdW5ncm91cCgpICU+JQogICMgU29tZSBzdWJqZWN0cyBtYXkgaGF2ZSBoYWQgbXVsdGlwbGUgImJlc3QiIG9wdGltaXphdGlvbiBydW5zCiAgIyBJbiB0aGF0IGNhc2UsIGp1c3QgZ28gd2l0aCB3aGljaGV2ZXIgImJlc3QiIHJ1biB3YXMgZXN0aW1hdGVkIGZpcnN0CiAgZ3JvdXBfYnkoc3ViX2lkKSAlPiUKICBzbGljZV9taW4ob3B0aW1pemVyX3J1biwgbiA9IDEpICU+JQogIHVuZ3JvdXAoKQoKcmVjb3ZlcnlfbGFwc2Vfc3IgPC0gbGFwc2VfcGFyYW1zX3RydWUgJT4lCiAgZmlsdGVyKG1vZGVsID09ICJzciIpICU+JQogIHBpdm90X3dpZGVyKAogICAgbmFtZXNfZnJvbSA9IHBhcmFtX25hbWUsCiAgICB2YWx1ZXNfZnJvbSA9IHBhcmFtX3ZhbHVlLAogICAgbmFtZXNfcHJlZml4ID0gInRydWVfIgogICkgJT4lCiAgbGVmdF9qb2luKAogICAgbGFwc2VfcGFyYW1zX2VzdF9zciAlPiUKICAgICAgc2VsZWN0KHN1Yl9pZCwgcGFyYW1fbmFtZSwgcGFyYW1fdmFsdWUgPSBwYXJhbV92YWx1ZV9odW1hbl9yZWFkYWJsZSkgJT4lCiAgICAgIHBpdm90X3dpZGVyKAogICAgICAgIG5hbWVzX2Zyb20gPSBwYXJhbV9uYW1lLAogICAgICAgIHZhbHVlc19mcm9tID0gcGFyYW1fdmFsdWUsCiAgICAgICAgbmFtZXNfcHJlZml4ID0gImVzdF8iCiAgICAgICksCiAgICBieSA9IGpvaW5fYnkoc3ViX2lkKQogICkKCnBsb3RfcmVjb3Zlcnlfc3JfbGFwc2VfZ2FtbWEgPC0gcmVjb3ZlcnlfbGFwc2Vfc3IgJT4lCiAgZ2dwbG90KGFlcyh4PXRydWVfc3JfZ2FtbWEsIHk9ZXN0X3NyX2dhbW1hKSkgKwogIHRoZW1lX2N1c3RvbSgpICsKICBnZW9tX2FibGluZShzbG9wZSA9IDEsIGludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRhc2hlZCIsIGNvbG9yID0gImJsdWUiKSArCiAgZ2VvbV9wb2ludChhbHBoYSA9IDAuMjUpICsKICB4bGFiKCJUcnVlIFNSIGdhbW1hIikgKwogIHlsYWIoIkVzdGltYXRlZCBTUiBnYW1tYSIpICsKICBnZ3RpdGxlKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IFNSIHcvIGxhcHNlIHJhdGUiKQoKcGxvdF9yZWNvdmVyeV9zcl9sYXBzZV9nYW1tYQoKcmVjb3ZlcnlfbGFwc2Vfc3IgJT4lCiAgd2l0aChjb3IudGVzdCh0cnVlX3NyX2dhbW1hLCBlc3Rfc3JfZ2FtbWEsIG1ldGhvZCA9ICJzcGVhcm1hbiIpKSAlPiUKICB0aWR5KCkgJT4lCiAga2FibGVfY3VzdG9tKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IFNSIHcvIGxhcHNlIHJhdGUiKQpgYGAKClRoZSByZWNvdmVyeSBvZiB0aGUgc29mdG1heCB0ZW1wZXJhdHVyZSBwYXJhbWV0ZXIgaXMgYWxzbyBmYWlybHkgYmFkLiBXZSBub3RlIHRoYXQgc29tZSBvZiB0aGUgc29mdG1heCB0ZW1wZXJhdHVyZXMgYXJlIGVzdGltYXRlZCB0byBiZSwgb24gYW4gYWJzb2x1dGUgc2NhbGUsIHZlcnkgbGFyZ2UuCgpgYGB7ciByZWNvdmVyeS1sYXBzZS1zci1zb2Z0bWF4fQpyZWNvdmVyeV9sYXBzZV9zciAlPiUKICBnZ3Bsb3QoYWVzKHg9dHJ1ZV9zb2Z0bWF4X3RlbXBlcmF0dXJlLCB5PWVzdF9zb2Z0bWF4X3RlbXBlcmF0dXJlKSkgKwogIHRoZW1lX2N1c3RvbSgpICsKICBnZW9tX2FibGluZShzbG9wZSA9IDEsIGludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRhc2hlZCIsIGNvbG9yID0gImJsdWUiKSArCiAgZ2VvbV9wb2ludChhbHBoYSA9IDAuMjUpICsKICB4bGFiKCJUcnVlIHNvZnRtYXggdGVtcGVyYXR1cmUiKSArCiAgeWxhYigiRXN0aW1hdGVkIHNvZnRtYXggdGVtcGVyYXR1cmUiKSArCiAgZ2d0aXRsZSgiUGFyYW1ldGVyIHJlY292ZXJ5OiBTUiB3LyBsYXBzZSByYXRlIikKCnJlY292ZXJ5X2xhcHNlX3NyICU+JQogIGdncGxvdChhZXMoeD10cnVlX3NvZnRtYXhfdGVtcGVyYXR1cmUsIHk9ZXN0X3NvZnRtYXhfdGVtcGVyYXR1cmUpKSArCiAgdGhlbWVfY3VzdG9tKCkgKwogIGdlb21fYWJsaW5lKHNsb3BlID0gMSwgaW50ZXJjZXB0ID0gMCwgbGluZXR5cGUgPSAiZGFzaGVkIiwgY29sb3IgPSAiYmx1ZSIpICsKICBnZW9tX3BvaW50KGFscGhhID0gMC4yNSkgKwogIHhsYWIoIlRydWUgc29mdG1heCB0ZW1wZXJhdHVyZSIpICsKICB5bGFiKCJFc3RpbWF0ZWQgc29mdG1heCB0ZW1wZXJhdHVyZSIpICsKICBjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoLTUwMDAwLCA1MDAwMCkpICsKICBnZ3RpdGxlKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IFNSIHcvIGxhcHNlIHJhdGUiKQoKcGxvdF9yZWNvdmVyeV9zcl9sYXBzZV9zb2Z0bWF4IDwtIHJlY292ZXJ5X2xhcHNlX3NyICU+JQogIGdncGxvdChhZXMoeD10cnVlX3NvZnRtYXhfdGVtcGVyYXR1cmUsIHk9ZXN0X3NvZnRtYXhfdGVtcGVyYXR1cmUpKSArCiAgdGhlbWVfY3VzdG9tKCkgKwogIGdlb21fYWJsaW5lKHNsb3BlID0gMSwgaW50ZXJjZXB0ID0gMCwgbGluZXR5cGUgPSAiZGFzaGVkIiwgY29sb3IgPSAiYmx1ZSIpICsKICBnZW9tX3BvaW50KGFscGhhID0gMC4yNSkgKwogIHhsYWIoIlRydWUgc29mdG1heCB0ZW1wZXJhdHVyZSIpICsKICB5bGFiKCJFc3RpbWF0ZWQgc29mdG1heCB0ZW1wZXJhdHVyZSIpICsKICBjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoLTUwMDAsIDUwMDApKSArCiAgZ2d0aXRsZSgiUGFyYW1ldGVyIHJlY292ZXJ5OiBTUiB3LyBsYXBzZSByYXRlIikKCnBsb3RfcmVjb3Zlcnlfc3JfbGFwc2Vfc29mdG1heAoKcmVjb3ZlcnlfbGFwc2Vfc3IgJT4lCiAgd2l0aCgKICAgIGNvci50ZXN0KAogICAgICB0cnVlX3NvZnRtYXhfdGVtcGVyYXR1cmUsIGVzdF9zb2Z0bWF4X3RlbXBlcmF0dXJlLAogICAgICBtZXRob2QgPSAic3BlYXJtYW4iCiAgICApCiAgKSAlPiUKICB0aWR5KCkgJT4lCiAga2FibGVfY3VzdG9tKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IFNSIHcvIGxhcHNlIHJhdGUiKQpgYGAKCkFuZCB0byByb3VuZCBldmVyeXRoaW5nIG91dCwgd2UgY2FuIHNlZSB0aGF0IHdlJ3JlIHVuYWJsZSB0byByZWxpYWJseSByZWNvdmVyIHRoZSBsYXBzZSByYXRlLgoKYGBge3IgcmVjb3ZlcnktbGFwc2Utc3ItbGFwc2V9CnBsb3RfcmVjb3Zlcnlfc3JfbGFwc2VfbGFwc2UgPC0gcmVjb3ZlcnlfbGFwc2Vfc3IgJT4lCiAgZ2dwbG90KGFlcyh4PXRydWVfbGFwc2VfcmF0ZSwgeT1lc3RfbGFwc2VfcmF0ZSkpICsKICB0aGVtZV9jdXN0b20oKSArCiAgZ2VvbV9hYmxpbmUoc2xvcGUgPSAxLCBpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJibHVlIikgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAwLjI1KSArCiAgeGxhYigiVHJ1ZSBsYXBzZSByYXRlIikgKwogIHlsYWIoIkVzdGltYXRlZCBsYXBzZSByYXRlIikgKwogIGdndGl0bGUoIlBhcmFtZXRlciByZWNvdmVyeTogU1Igdy8gbGFwc2UgcmF0ZSIpCgpwbG90X3JlY292ZXJ5X3NyX2xhcHNlX2xhcHNlCgpyZWNvdmVyeV9sYXBzZV9zciAlPiUKICB3aXRoKGNvci50ZXN0KHRydWVfbGFwc2VfcmF0ZSwgZXN0X2xhcHNlX3JhdGUsIG1ldGhvZCA9ICJzcGVhcm1hbiIpKSAlPiUKICB0aWR5KCkgJT4lCiAga2FibGVfY3VzdG9tKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IFNSIHcvIGxhcHNlIHJhdGUiKQpgYGAKCkxldCdzIGNyZWF0ZSBhIHZpc3VhbGl6YXRpb24gY29tYmluaW5nIHRoZXNlIHBsb3RzOgoKYGBge3IgcGxvdC1yZWNvdmVyeS1sYXBzZS1zcn0KI3wgZmlnLndpZHRoPTgsIGZpZy5oZWlnaHQ9NAoKcGxvdF9yZWNvdmVyeV9zcl9sYXBzZV9jb21iaW5lZCA8LSB3cmFwX3Bsb3RzKAogIHBsb3RfcmVjb3Zlcnlfc3JfbGFwc2VfZ2FtbWEgKyBnZ3RpdGxlKCJHYW1tYSIpLAogIHBsb3RfcmVjb3Zlcnlfc3JfbGFwc2Vfc29mdG1heCArIGdndGl0bGUoIlNvZnRtYXggdGVtcGVyYXR1cmUiKSwKICBwbG90X3JlY292ZXJ5X3NyX2xhcHNlX2xhcHNlICsgZ2d0aXRsZSgiTGFwc2UgcmF0ZSIpCikgKwogIHBsb3RfYW5ub3RhdGlvbigKICAgIHRpdGxlID0gIlBhcmFtZXRlciByZWNvdmVyeTogU3VjY2Vzc29yIFJlcHJlc2VudGF0aW9uIHdpdGggbGFwc2UgcmF0ZSIsCiAgICB0aGVtZSA9IHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSwKICAgIHRhZ19sZXZlbHMgPSAiQSIsCiAgICB0YWdfc3VmZml4ID0gIi4iCiAgKSAmCiAgeGxhYigiVHJ1ZSBwYXJhbWV0ZXIgdmFsdWUiKSAmCiAgeWxhYigiRXN0aW1hdGVkIHBhcmFtZXRlciB2YWx1ZSIpCgpwbG90X3JlY292ZXJ5X3NyX2xhcHNlX2NvbWJpbmVkCgppZiAoa25pdHRpbmcpIHsKICBnZ3NhdmUoCiAgICBmaWxlbmFtZSA9IGhlcmUoIm91dHB1dHMiLCB3b3JrZmxvd19uYW1lLCAicGFyYW1fcmVjb3Zlcnlfc3JfbGFwc2UucGRmIiksCiAgICBwbG90ID0gcGxvdF9yZWNvdmVyeV9zcl9sYXBzZV9jb21iaW5lZCwKICAgIHdpZHRoID0gOCwgaGVpZ2h0ID0gNCwKICAgIHVuaXRzID0gImluIiwgZHBpID0gMzAwCiAgKQp9CmBgYAoKIyMgQkZTLWZvcndhcmQKCldlIGhhZCBwcmV2aW91c2x5IGFsc28gZXN0aW1hdGVkIGEgbW9kZWwgb2YgKGZvcndhcmQpLUJGUyB3aXRoIGEgbGFwc2UgcmF0ZS4gSGVyZSwgd2UnbGwgdGVzdCB0aGUgcmVjb3ZlcmFiaWxpdHkgb2YgdGhpcyBtb2RlbC4KClJlc3VsdHMgaW5kaWNhdGUgcmVjb3Zlcnkgb2YgdGhlIHNlYXJjaCB0aHJlc2hvbGQgcGFyYW1ldGVyIGlzIG1lZGlvY3JlLgoKYGBge3IgcmVjb3ZlcnktbGFwc2UtYmZzLWZvcndhcmQtc2VhcmNoLXRocmVzaG9sZH0KbGFwc2VfcGFyYW1zX2VzdF9iZnNfZm9yd2FyZCA8LSBoZXJlKAogICJkYXRhIiwgInBhcmFtX3JlY292ZXJ5IiwgImJmc19mb3J3YXJkX3dpdGhfbGFwc2UiCikgJT4lCiAgZnM6OmRpcl9scyhnbG9iID0gIiouY3N2IikgJT4lCiAgbWFwX2RmciguZiA9IH5yZWFkX2NzdigueCwgc2hvd19jb2xfdHlwZXMgPSBGQUxTRSksIC5pZCA9ICJmaWxlbmFtZSIpICU+JQogIG11dGF0ZSgKICAgIHN1Yl9pZCA9IHN0cl9leHRyYWN0KGZpbGVuYW1lLCAic3ViX1tbOmRpZ2l0Ol1dKyIpLAogICAgc3ViX2lkID0gc3RyX3JlbW92ZShzdWJfaWQsICJzdWJfIiksCiAgICBzdWJfaWQgPSBhcy5udW1lcmljKHN1Yl9pZCkKICApICU+JQogIHNlbGVjdChzdWJfaWQsIGV2ZXJ5dGhpbmcoKSwgLWZpbGVuYW1lKSAlPiUKICByZW5hbWUobmVnX2xvZ2xpayA9IG9wdGltX3ZhbHVlKSAlPiUKICAjIEZpbmQgYmVzdC1maXR0aW5nIG9wdGltaXphdGlvbiBydW4KICBmaWx0ZXIoY29udmVyZ2VuY2UgPT0gImNvbnZlcmdlZCIpICU+JQogIGdyb3VwX2J5KHN1Yl9pZCkgJT4lCiAgc2xpY2VfbWluKG5lZ19sb2dsaWssIG4gPSAxKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgIyBTb21lIHN1YmplY3RzIG1heSBoYXZlIGhhZCBtdWx0aXBsZSAiYmVzdCIgb3B0aW1pemF0aW9uIHJ1bnMKICAjIEluIHRoYXQgY2FzZSwganVzdCBnbyB3aXRoIHdoaWNoZXZlciAiYmVzdCIgcnVuIHdhcyBlc3RpbWF0ZWQgZmlyc3QKICBncm91cF9ieShzdWJfaWQpICU+JQogIHNsaWNlX21pbihvcHRpbWl6ZXJfcnVuLCBuID0gMSkgJT4lCiAgdW5ncm91cCgpCgpyZWNvdmVyeV9sYXBzZV9iZnNfZm9yd2FyZCA8LSBsYXBzZV9wYXJhbXNfdHJ1ZSAlPiUKICBmaWx0ZXIobW9kZWwgPT0gImJmc19mb3J3YXJkIikgJT4lCiAgcGl2b3Rfd2lkZXIoCiAgICBuYW1lc19mcm9tID0gcGFyYW1fbmFtZSwKICAgIHZhbHVlc19mcm9tID0gcGFyYW1fdmFsdWUsCiAgICBuYW1lc19wcmVmaXggPSAidHJ1ZV8iCiAgKSAlPiUKICBsZWZ0X2pvaW4oCiAgICBsYXBzZV9wYXJhbXNfZXN0X2Jmc19mb3J3YXJkICU+JQogICAgICBzZWxlY3Qoc3ViX2lkLCBwYXJhbV9uYW1lLCBwYXJhbV92YWx1ZSA9IHBhcmFtX3ZhbHVlX2h1bWFuX3JlYWRhYmxlKSAlPiUKICAgICAgcGl2b3Rfd2lkZXIoCiAgICAgICAgbmFtZXNfZnJvbSA9IHBhcmFtX25hbWUsCiAgICAgICAgdmFsdWVzX2Zyb20gPSBwYXJhbV92YWx1ZSwKICAgICAgICBuYW1lc19wcmVmaXggPSAiZXN0XyIKICAgICAgKSwKICAgIGJ5ID0gam9pbl9ieShzdWJfaWQpCiAgKQoKcGxvdF9yZWNvdmVyeV9iZnNfZm9yd2FyZF9sYXBzZV90aHJlc2hvbGQgPC0gcmVjb3ZlcnlfbGFwc2VfYmZzX2ZvcndhcmQgJT4lCiAgZ2dwbG90KGFlcyh4PXRydWVfc2VhcmNoX3RocmVzaG9sZCwgeT1lc3Rfc2VhcmNoX3RocmVzaG9sZCkpICsKICB0aGVtZV9jdXN0b20oKSArCiAgZ2VvbV9hYmxpbmUoc2xvcGUgPSAxLCBpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJibHVlIikgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAwLjI1KSArCiAgeGxhYigiVHJ1ZSBzZWFyY2ggdGhyZXNob2xkIikgKwogIHlsYWIoIkVzdGltYXRlZCBzZWFyY2ggdGhyZXNob2xkIikgKwogIGdndGl0bGUoIlBhcmFtZXRlciByZWNvdmVyeTogQkZTLWZvcndhcmQgdy8gbGFwc2UgcmF0ZSIpCgpwbG90X3JlY292ZXJ5X2Jmc19mb3J3YXJkX2xhcHNlX3RocmVzaG9sZAoKcmVjb3ZlcnlfbGFwc2VfYmZzX2ZvcndhcmQgJT4lCiAgd2l0aCgKICAgIGNvci50ZXN0KHRydWVfc2VhcmNoX3RocmVzaG9sZCwgZXN0X3NlYXJjaF90aHJlc2hvbGQsIG1ldGhvZCA9ICJzcGVhcm1hbiIpCiAgKSAlPiUKICB0aWR5KCkgJT4lCiAga2FibGVfY3VzdG9tKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IEJGUy1mb3J3YXJkIHcvIGxhcHNlIHJhdGUiKQpgYGAKClRoZSByZWNvdmVyeSBvZiB0aGUgbGFwc2UgcmF0ZSBpcyBhbHNvIG1lZGlvY3JlLgoKYGBge3IgcmVjb3ZlcnktbGFwc2UtYmZzLWZvcndhcmQtbGFwc2UtcmF0ZX0KcGxvdF9yZWNvdmVyeV9iZnNfZm9yd2FyZF9sYXBzZV9sYXBzZSA8LSByZWNvdmVyeV9sYXBzZV9iZnNfZm9yd2FyZCAlPiUKICBnZ3Bsb3QoYWVzKHg9dHJ1ZV9sYXBzZV9yYXRlLCB5PWVzdF9sYXBzZV9yYXRlKSkgKwogIHRoZW1lX2N1c3RvbSgpICsKICBnZW9tX2FibGluZShzbG9wZSA9IDEsIGludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRhc2hlZCIsIGNvbG9yID0gImJsdWUiKSArCiAgZ2VvbV9wb2ludChhbHBoYSA9IDAuMjUpICsKICB4bGFiKCJUcnVlIGxhcHNlIikgKwogIHlsYWIoIkVzdGltYXRlZCBsYXBzZSByYXRlIikgKwogIGdndGl0bGUoIlBhcmFtZXRlciByZWNvdmVyeTogQkZTLWZvcndhcmQgdy8gbGFwc2UgcmF0ZSIpCgpwbG90X3JlY292ZXJ5X2Jmc19mb3J3YXJkX2xhcHNlX2xhcHNlCgpyZWNvdmVyeV9sYXBzZV9iZnNfZm9yd2FyZCAlPiUKICB3aXRoKGNvci50ZXN0KHRydWVfbGFwc2VfcmF0ZSwgZXN0X2xhcHNlX3JhdGUsIG1ldGhvZCA9ICJzcGVhcm1hbiIpKSAlPiUKICB0aWR5KCkgJT4lCiAga2FibGVfY3VzdG9tKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IEJGUy1mb3J3YXJkIHcvIGxhcHNlIHJhdGUiKQpgYGAKClBsb3R0aW5nIHRoZW0gdG9nZXRoZXIuLi4KCmBgYHtyIHBsb3QtcmVjb3ZlcnktbGFwc2UtYmZzLWZvcndhcmR9CiN8IGZpZy53aWR0aD04LCBmaWcuaGVpZ2h0PTQKCnBsb3RfcmVjb3ZlcnlfYmZzX2ZvcndhcmRfbGFwc2VfY29tYmluZWQgPC0gd3JhcF9wbG90cygKICBwbG90X3JlY292ZXJ5X2Jmc19mb3J3YXJkX2xhcHNlX3RocmVzaG9sZCArIGdndGl0bGUoIlNlYXJjaCB0aHJlc2hvbGQiKSwKICBwbG90X3JlY292ZXJ5X2Jmc19mb3J3YXJkX2xhcHNlX2xhcHNlICsgZ2d0aXRsZSgiTGFwc2UgcmF0ZSIpCikgKwogIHBsb3RfYW5ub3RhdGlvbigKICAgIHRpdGxlID0gIlBhcmFtZXRlciByZWNvdmVyeTogQkZTLWZvcndhcmQiLAogICAgdGhlbWUgPSB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSksCiAgICB0YWdfbGV2ZWxzID0gIkEiLAogICAgdGFnX3N1ZmZpeCA9ICIuIgogICkgJgogIHhsYWIoIlRydWUgcGFyYW1ldGVyIHZhbHVlIikgJgogIHlsYWIoIkVzdGltYXRlZCBwYXJhbWV0ZXIgdmFsdWUiKQoKcGxvdF9yZWNvdmVyeV9iZnNfZm9yd2FyZF9sYXBzZV9jb21iaW5lZAoKaWYgKGtuaXR0aW5nKSB7CiAgZ2dzYXZlKAogICAgZmlsZW5hbWUgPSBoZXJlKAogICAgICAib3V0cHV0cyIsIHdvcmtmbG93X25hbWUsICJwYXJhbV9yZWNvdmVyeV9iZnNfZm9yd2FyZF9sYXBzZS5wZGYiCiAgICApLAogICAgcGxvdCA9IHBsb3RfcmVjb3ZlcnlfYmZzX2ZvcndhcmRfbGFwc2VfY29tYmluZWQsCiAgICB3aWR0aCA9IDgsIGhlaWdodCA9IDQsCiAgICB1bml0cyA9ICJpbiIsIGRwaSA9IDMwMAogICkKfQpgYGAKCgojIE1vZGVscyB3aXRob3V0IGEgbGFwc2UgcmF0ZQoKU2luY2UgaXQgYXBwZWFycyB0aGF0IGxhcHNlIHJhdGVzIGNhbm5vdCBiZSByZWxpYWJseSByZWNvdmVyZWQsIHdlJ2xsIG5vdyB0cnkgbG9va2luZyBhdCBtb2RlbHMgdGhhdCBkb24ndCBjb250YWluIGEgbGFwc2UgcmF0ZS4gTm90ZSB0aGF0IHRoZXNlIGluY2x1ZGUgbmV3IG1vZGVscyB0aGF0IHdlJ3ZlIGFkZGVkIGluIHRoZSByZXZpc2lvbi4KCmBgYHtyIGxvYWQtcGFyYW1zLW5vLWxhcHNlfQpwYXJhbXNfdHJ1ZSA8LSBoZXJlKCJkYXRhIiwgInNpbXVsYXRlZF9tb2RlbF9iZWhhdmlvcnMiKSAlPiUKICBmczo6ZGlyX2xzKHJlZ2V4cCA9ICJub19sYXBzZVxcLmNzdiIpICU+JQogIG1hcF9kZnIoLmYgPSB+cmVhZF9jc3YoLngsIHNob3dfY29sX3R5cGVzID0gRkFMU0UpLCAuaWQgPSAiZmlsZW5hbWUiKSAlPiUKICBtdXRhdGUoCiAgICBtb2RlbCA9IHN0cl9leHRyYWN0KGZpbGVuYW1lLCAiYmZzXyhiYWNrd2FyZHxmb3J3YXJkKXxpZGVhbF9vYnN8c3IiKQogICkgJT4lCiAgc2VsZWN0KG1vZGVsLCBzdWJfaWQsIHNlYXJjaF90aHJlc2hvbGQsIHNvZnRtYXhfdGVtcGVyYXR1cmUsIHNyX2dhbW1hKSAlPiUKICBkaXN0aW5jdCgpICU+JQogIHBpdm90X2xvbmdlcigKICAgIC1jKG1vZGVsLCBzdWJfaWQpLCBuYW1lc190byA9ICJwYXJhbV9uYW1lIiwgdmFsdWVzX3RvID0gInBhcmFtX3ZhbHVlIgogICkgJT4lCiAgZHJvcF9uYSgpCgpwYXJhbXNfZXN0IDwtIGhlcmUoImRhdGEiLCAicGFyYW1fcmVjb3ZlcnkiKSAlPiUKICBmczo6ZGlyX2xzKAogICAgcmVnZXhwID0gIihiZnNfKGJhY2t3YXJkfGZvcndhcmQpfGlkZWFsX29ic3xzcilfbm9fbGFwc2UoLikrXFwuY3N2IiwKICAgIHJlY3Vyc2UgPSAxCiAgKSAlPiUKICBtYXBfZGZyKC5mID0gfnJlYWRfY3N2KC54LCBzaG93X2NvbF90eXBlcyA9IEZBTFNFKSwgLmlkID0gImZpbGVuYW1lIikgJT4lCiAgbXV0YXRlKAogICAgc3ViX2lkID0gc3RyX2V4dHJhY3QoZmlsZW5hbWUsICJzdWJfW1s6ZGlnaXQ6XV0rIiksCiAgICBzdWJfaWQgPSBzdHJfcmVtb3ZlKHN1Yl9pZCwgInN1Yl8iKSwKICAgIHN1Yl9pZCA9IGFzLm51bWVyaWMoc3ViX2lkKSwKICAgIG1vZGVsID0gc3RyX2V4dHJhY3QoZmlsZW5hbWUsICJiZnNfKGJhY2t3YXJkfGZvcndhcmQpfGlkZWFsX29ic3xzciIpCiAgKSAlPiUKICBzZWxlY3Qoc3ViX2lkLCBldmVyeXRoaW5nKCksIC1maWxlbmFtZSkgJT4lCiAgcmVuYW1lKG5lZ19sb2dsaWsgPSBvcHRpbV92YWx1ZSkgJT4lCiAgIyBGaW5kIGJlc3QtZml0dGluZyBvcHRpbWl6YXRpb24gcnVuCiAgZmlsdGVyKGNvbnZlcmdlbmNlID09ICJjb252ZXJnZWQiKSAlPiUKICBncm91cF9ieShzdWJfaWQsIG1vZGVsKSAlPiUKICBzbGljZV9taW4obmVnX2xvZ2xpaywgbiA9IDEpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAjIFNvbWUgc3ViamVjdHMgbWF5IGhhdmUgaGFkIG11bHRpcGxlICJiZXN0IiBvcHRpbWl6YXRpb24gcnVucwogICMgSW4gdGhhdCBjYXNlLCBqdXN0IGdvIHdpdGggd2hpY2hldmVyICJiZXN0IiBydW4gd2FzIGVzdGltYXRlZCBmaXJzdAogIGdyb3VwX2J5KHN1Yl9pZCwgbW9kZWwpICU+JQogIHNsaWNlX21pbihvcHRpbWl6ZXJfcnVuLCBuID0gMSkgJT4lCiAgdW5ncm91cCgpICU+JQogICMgQ2xlYW4gdXAKICBtdXRhdGUoCiAgICBwYXJhbV92YWx1ZV90b19rZWVwID0gaWZfZWxzZSgKICAgICAgaXMubmEocGFyYW1fdmFsdWUpLCBwYXJhbV92YWx1ZV9odW1hbl9yZWFkYWJsZSwgcGFyYW1fdmFsdWUKICAgICkKICApICU+JQogIHNlbGVjdCgKICAgIG1vZGVsLCBzdWJfaWQsIHBhcmFtX25hbWUsIHBhcmFtX3ZhbHVlID0gcGFyYW1fdmFsdWVfdG9fa2VlcCwgbmVnX2xvZ2xpawogICkgJT4lCiAgYXJyYW5nZShtb2RlbCwgc3ViX2lkLCBwYXJhbV9uYW1lKQpgYGAKCiMjIEJGUy1iYWNrd2FyZAoKUmVjb3ZlcnkgaXMgZ2VuZXJhbGx5IHZlcnkgZ29vZC4gV2Ugbm90ZSB0aGF0IHRoZSBvcHRpbWl6ZXIgdGVuZHMgdG8gY29udmVyZ2UgdXBvbiB0d28gKGluYWNjdXJhdGUpIG1vZGVzIHdoZW4gdGhlIHRydWUgc2VhcmNoIHRocmVzaG9sZCBpcyBncmVhdGVyIHRoYW4gMTAuIFdlIGtub3cgdGhhdCBiZWhhdmlvciBiYXNpY2FsbHkgYXN5bXB0b3RlcyBhcm91bmQgdGhlbiwgc28gZXN0aW1hdGVkIHRocmVzaG9sZHMgZ3JlYXRlciB0aGFuIDEwIHNob3VsZCBwcm9iYWJseSBiZSB0cmVhdGVkIGFzIGJlaW5nIGluZGlzdGluZ3Vpc2hhYmxlIGZyb20gYXN5bXB0b3RpYy4KCmBgYHtyIHJlY292ZXJ5LWJmcy1iYWNrd2FyZH0KcmVjb3ZlcnlfYmZzX2JhY2t3YXJkIDwtIHBhcmFtc190cnVlICU+JQogIGZpbHRlcihtb2RlbCA9PSAiYmZzX2JhY2t3YXJkIikgJT4lCiAgcGl2b3Rfd2lkZXIoCiAgICBuYW1lc19mcm9tID0gcGFyYW1fbmFtZSwKICAgIHZhbHVlc19mcm9tID0gcGFyYW1fdmFsdWUsCiAgICBuYW1lc19wcmVmaXggPSAidHJ1ZV8iCiAgKSAlPiUKICBsZWZ0X2pvaW4oCiAgICBwYXJhbXNfZXN0ICU+JQogICAgICBmaWx0ZXIobW9kZWwgPT0gImJmc19iYWNrd2FyZCIpICU+JQogICAgICBzZWxlY3Qoc3ViX2lkLCBwYXJhbV9uYW1lLCBwYXJhbV92YWx1ZSkgJT4lCiAgICAgIHBpdm90X3dpZGVyKAogICAgICAgIG5hbWVzX2Zyb20gPSBwYXJhbV9uYW1lLAogICAgICAgIHZhbHVlc19mcm9tID0gcGFyYW1fdmFsdWUsCiAgICAgICAgbmFtZXNfcHJlZml4ID0gImVzdF8iCiAgICAgICksCiAgICBieSA9IGpvaW5fYnkoc3ViX2lkKQogICkKCnBsb3RfcmVjb3ZlcnlfYmZzX2JhY2t3YXJkIDwtIHJlY292ZXJ5X2Jmc19iYWNrd2FyZCAlPiUKICBnZ3Bsb3QoYWVzKHg9dHJ1ZV9zZWFyY2hfdGhyZXNob2xkLCB5PWVzdF9zZWFyY2hfdGhyZXNob2xkKSkgKwogIHRoZW1lX2N1c3RvbSgpICsKICBnZW9tX2FibGluZShzbG9wZSA9IDEsIGludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRhc2hlZCIsIGNvbG9yID0gImJsdWUiKSArCiAgZ2VvbV9wb2ludChhbHBoYSA9IDAuMjUpICsKICB4bGFiKCJUcnVlIHNlYXJjaCB0aHJlc2hvbGQiKSArCiAgeWxhYigiRXN0aW1hdGVkIHNlYXJjaCB0aHJlc2hvbGQiKSArCiAgZ2d0aXRsZSgiUGFyYW1ldGVyIHJlY292ZXJ5OiBCRlMtYmFja3dhcmQiKQoKcGxvdF9yZWNvdmVyeV9iZnNfYmFja3dhcmQKCnJlY292ZXJ5X2Jmc19iYWNrd2FyZCAlPiUKICB3aXRoKAogICAgY29yLnRlc3QodHJ1ZV9zZWFyY2hfdGhyZXNob2xkLCBlc3Rfc2VhcmNoX3RocmVzaG9sZCwgbWV0aG9kID0gInNwZWFybWFuIikKICApICU+JQogIHRpZHkoKSAlPiUKICBrYWJsZV9jdXN0b20oIlBhcmFtZXRlciByZWNvdmVyeTogQkZTLWJhY2t3YXJkIikKYGBgCgojIyBCRlMtZm9yd2FyZAoKUmVjb3ZlcnkgaXMgc2ltaWxhcmx5IGdvb2QgZm9yIHRoZSBCRlMtZm9yd2FyZCBtb2RlbCwgdGhvdWdoIHdlIG5vdGUgdGhhdCB0aGUgb3B0aW1pemVyIHNvbWV0aW1lcyBvdmVyZXN0aW1hdGVzIGxhcmdlIHRocmVzaG9sZHMgKHN0YXJ0aW5nIGF0IGFyb3VuZCAxMikgYW5kIHVuZGVyZXN0aW1hdGVzIHNtYWxsIHRocmVzaG9sZHMgKGVuZGluZyBhdCBhcm91bmQgNCkuCgpgYGB7ciByZWNvdmVyeS1iZnMtZm9yd2FyZH0KcmVjb3ZlcnlfYmZzX2ZvcndhcmQgPC0gcGFyYW1zX3RydWUgJT4lCiAgZmlsdGVyKG1vZGVsID09ICJiZnNfZm9yd2FyZCIpICU+JQogIHBpdm90X3dpZGVyKAogICAgbmFtZXNfZnJvbSA9IHBhcmFtX25hbWUsCiAgICB2YWx1ZXNfZnJvbSA9IHBhcmFtX3ZhbHVlLAogICAgbmFtZXNfcHJlZml4ID0gInRydWVfIgogICkgJT4lCiAgbGVmdF9qb2luKAogICAgcGFyYW1zX2VzdCAlPiUKICAgICAgZmlsdGVyKG1vZGVsID09ICJiZnNfZm9yd2FyZCIpICU+JQogICAgICBzZWxlY3Qoc3ViX2lkLCBwYXJhbV9uYW1lLCBwYXJhbV92YWx1ZSkgJT4lCiAgICAgIHBpdm90X3dpZGVyKAogICAgICAgIG5hbWVzX2Zyb20gPSBwYXJhbV9uYW1lLAogICAgICAgIHZhbHVlc19mcm9tID0gcGFyYW1fdmFsdWUsCiAgICAgICAgbmFtZXNfcHJlZml4ID0gImVzdF8iCiAgICAgICksCiAgICBieSA9IGpvaW5fYnkoc3ViX2lkKQogICkKCnBsb3RfcmVjb3ZlcnlfYmZzX2ZvcndhcmQgPC0gcmVjb3ZlcnlfYmZzX2ZvcndhcmQgJT4lCiAgZ2dwbG90KGFlcyh4PXRydWVfc2VhcmNoX3RocmVzaG9sZCwgeT1lc3Rfc2VhcmNoX3RocmVzaG9sZCkpICsKICB0aGVtZV9jdXN0b20oKSArCiAgZ2VvbV9hYmxpbmUoc2xvcGUgPSAxLCBpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJibHVlIikgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAwLjI1KSArCiAgeGxhYigiVHJ1ZSBzZWFyY2ggdGhyZXNob2xkIikgKwogIHlsYWIoIkVzdGltYXRlZCBzZWFyY2ggdGhyZXNob2xkIikgKwogIGdndGl0bGUoIlBhcmFtZXRlciByZWNvdmVyeTogQkZTLWZvcndhcmQiKQoKcGxvdF9yZWNvdmVyeV9iZnNfZm9yd2FyZAoKcmVjb3ZlcnlfYmZzX2ZvcndhcmQgJT4lCiAgd2l0aCgKICAgIGNvci50ZXN0KHRydWVfc2VhcmNoX3RocmVzaG9sZCwgZXN0X3NlYXJjaF90aHJlc2hvbGQsIG1ldGhvZCA9ICJzcGVhcm1hbiIpCiAgKSAlPiUKICB0aWR5KCkgJT4lCiAga2FibGVfY3VzdG9tKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IEJGUy1mb3J3YXJkIikKYGBgCgojIyBJZGVhbCBvYnNlcnZlcgoKUGFyYW1ldGVyIHJlY292ZXJ5IGZvciB0aGUgaWRlYWwgb2JzZXJ2ZXIgbW9kZWwgaXMgY2xlYXJseSBiaWFzZWQsIHN1Y2ggdGhhdCB0aGUgb3B0aW1pemVyIGlzIGJlc3QgYXQgcmVjb3ZlcmluZyAoaW52ZXJzZSkgdGVtcGVyYXR1cmVzIGluIHRoZSByYW5nZSBvZiBbLTEsIDBdLCBsZXNzIGdvb2QgYXQgcmVjb3ZlcmluZyB0ZW1wZXJhdHVyZXMgaW4gdGhlIHJhbmdlIG9mIFstMiwgLTFdLCBhbmQgbXVjaCB3b3JzZSBiZXlvbmQgdGhhdC4gSXQgbWF5IGJlIHdpc2UgdG8gaW50ZXJwcmV0IGVzdGltYXRlZCB0ZW1wZXJhdHVyZXMgYmV5b25kIC0yIGFzIGJlaW5nICJ2ZXJ5IGNvbnNpc3RlbnQiIHdpdGggYW4gaWRlYWwgb2JzZXJ2ZXIgbW9kZWwuCgpgYGB7ciByZWNvdmVyeS1pZGVhbC1vYnN9CnJlY292ZXJ5X2lkZWFsX29icyA8LSBwYXJhbXNfdHJ1ZSAlPiUKICBmaWx0ZXIobW9kZWwgPT0gImlkZWFsX29icyIpICU+JQogIHBpdm90X3dpZGVyKAogICAgbmFtZXNfZnJvbSA9IHBhcmFtX25hbWUsCiAgICB2YWx1ZXNfZnJvbSA9IHBhcmFtX3ZhbHVlLAogICAgbmFtZXNfcHJlZml4ID0gInRydWVfIgogICkgJT4lCiAgbGVmdF9qb2luKAogICAgcGFyYW1zX2VzdCAlPiUKICAgICAgZmlsdGVyKG1vZGVsID09ICJpZGVhbF9vYnMiKSAlPiUKICAgICAgc2VsZWN0KHN1Yl9pZCwgcGFyYW1fbmFtZSwgcGFyYW1fdmFsdWUpICU+JQogICAgICBwaXZvdF93aWRlcigKICAgICAgICBuYW1lc19mcm9tID0gcGFyYW1fbmFtZSwKICAgICAgICB2YWx1ZXNfZnJvbSA9IHBhcmFtX3ZhbHVlLAogICAgICAgIG5hbWVzX3ByZWZpeCA9ICJlc3RfIgogICAgICApLAogICAgYnkgPSBqb2luX2J5KHN1Yl9pZCkKICApCgpwbG90X3JlY292ZXJ5X2lkZWFsX29icyA8LSByZWNvdmVyeV9pZGVhbF9vYnMgJT4lCiAgZ2dwbG90KGFlcyh4PXRydWVfc29mdG1heF90ZW1wZXJhdHVyZSwgeT1lc3Rfc29mdG1heF90ZW1wZXJhdHVyZSkpICsKICB0aGVtZV9jdXN0b20oKSArCiAgZ2VvbV9hYmxpbmUoc2xvcGUgPSAxLCBpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJibHVlIikgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAwLjI1KSArCiAgeGxhYigiVHJ1ZSBzb2Z0bWF4IHRlbXBlcmF0dXJlIikgKwogIHlsYWIoIkVzdGltYXRlZCBzb2Z0bWF4IHRlbXBlcmF0dXJlIikgKwogIGdndGl0bGUoIlBhcmFtZXRlciByZWNvdmVyeTogSWRlYWwgb2JzZXJ2ZXIiKQoKcGxvdF9yZWNvdmVyeV9pZGVhbF9vYnMKCnJlY292ZXJ5X2lkZWFsX29icyAlPiUKICB3aXRoKAogICAgY29yLnRlc3QoCiAgICAgIHRydWVfc29mdG1heF90ZW1wZXJhdHVyZSwgZXN0X3NvZnRtYXhfdGVtcGVyYXR1cmUsCiAgICAgIG1ldGhvZCA9ICJzcGVhcm1hbiIKICAgICkKICApICU+JQogIHRpZHkoKSAlPiUKICBrYWJsZV9jdXN0b20oIlBhcmFtZXRlciByZWNvdmVyeTogSWRlYWwgb2JzZXJ2ZXIiKQpgYGAKCiMjIFN1Y2Nlc3NvciBSZXByZXNlbnRhdGlvbgoKV2UgZmluZCB0aGF0IHRoZSBTUiBnYW1tYSBwYXJhbWV0ZXIgaGFzIGFjY2VwdGFibGUgcmVjb3ZlcmFiaWxpdHksIHRob3VnaCB3ZSBhbHNvIG5vdGUgdGhhdCBzbWFsbGVyIHZhbHVlcyBvZiBnYW1tYSBhcHBlYXIgdG8gYmUgYSBsaXR0bGUgbGVzcyByZWxpYWJseSByZWNvdmVyZWQ6IHRydWUgZ2FtbWFzIGJlbG93IDAuMjUgY2FuIGJlIHVuZGVyZXN0aW1hdGVkIGFzIG5lYXItemVyby4KCmBgYHtyIHJlY292ZXJ5LXNyLWdhbW1hfQpyZWNvdmVyeV9zciA8LSBwYXJhbXNfdHJ1ZSAlPiUKICBmaWx0ZXIobW9kZWwgPT0gInNyIikgJT4lCiAgcGl2b3Rfd2lkZXIoCiAgICBuYW1lc19mcm9tID0gcGFyYW1fbmFtZSwKICAgIHZhbHVlc19mcm9tID0gcGFyYW1fdmFsdWUsCiAgICBuYW1lc19wcmVmaXggPSAidHJ1ZV8iCiAgKSAlPiUKICBsZWZ0X2pvaW4oCiAgICBwYXJhbXNfZXN0ICU+JQogICAgICBmaWx0ZXIobW9kZWwgPT0gInNyIikgJT4lCiAgICAgIHNlbGVjdChzdWJfaWQsIHBhcmFtX25hbWUsIHBhcmFtX3ZhbHVlKSAlPiUKICAgICAgcGl2b3Rfd2lkZXIoCiAgICAgICAgbmFtZXNfZnJvbSA9IHBhcmFtX25hbWUsCiAgICAgICAgdmFsdWVzX2Zyb20gPSBwYXJhbV92YWx1ZSwKICAgICAgICBuYW1lc19wcmVmaXggPSAiZXN0XyIKICAgICAgKSwKICAgIGJ5ID0gam9pbl9ieShzdWJfaWQpCiAgKQoKcGxvdF9yZWNvdmVyeV9zcl9nYW1tYSA8LSByZWNvdmVyeV9zciAlPiUKICBnZ3Bsb3QoYWVzKHg9dHJ1ZV9zcl9nYW1tYSwgeT1lc3Rfc3JfZ2FtbWEpKSArCiAgdGhlbWVfY3VzdG9tKCkgKwogIGdlb21fYWJsaW5lKHNsb3BlID0gMSwgaW50ZXJjZXB0ID0gMCwgbGluZXR5cGUgPSAiZGFzaGVkIiwgY29sb3IgPSAiYmx1ZSIpICsKICBnZW9tX3BvaW50KGFscGhhID0gMC4yNSkgKwogIHhsYWIoIlRydWUgZ2FtbWEiKSArCiAgeWxhYigiRXN0aW1hdGVkIGdhbW1hIikgKwogIGdndGl0bGUoIlBhcmFtZXRlciByZWNvdmVyeTogU3VjY2Vzc29yIFJlcC4iKQoKcGxvdF9yZWNvdmVyeV9zcl9nYW1tYQoKcmVjb3Zlcnlfc3IgJT4lCiAgd2l0aChjb3IudGVzdCh0cnVlX3NyX2dhbW1hLCBlc3Rfc3JfZ2FtbWEsIG1ldGhvZCA9ICJzcGVhcm1hbiIpKSAlPiUKICB0aWR5KCkgJT4lCiAga2FibGVfY3VzdG9tKCJQYXJhbWV0ZXIgcmVjb3Zlcnk6IFN1Y2Nlc3NvciBSZXAuIikKYGBgCgpSZWNvdmVyeSBvZiB0aGUgc29mdG1heCB0ZW1wZXJhdHVyZSBwYXJhbWV0ZXIgaXMgbGVzcyBnb29kLiBXZSBjYW4gc2VlIHRoYXQgdGhlIG9wdGltaXplciwgb24gb2NjYXNpb24sIGVzdGltYXRlcyBleHRyZW1lbHkgbGFyZ2UgdmFsdWVzIG9mIHRoaXMgcGFyYW1ldGVyLiBCZWxvdywgd2UgcGxvdCBhbGwgb2YgdGhlIGRhdGEgaW4gdGhlIGZpcnN0IGdyYXBoLCB0aGVuIHpvb20gaW4gdG8gc2VlIG1vcmUgdHlwaWNhbCB2YWx1ZXMgaW4gdGhlIHNlY29uZCBncmFwaC4gVGhlIHRoaXJkIGdyYXBoIHpvb21zIGluIGEgYml0IG1vcmUgdG8gZW1waGFzaXplIHRoYXQgcGFzdCB0cnVlIHNvZnRtYXggdGVtcGVyYXR1cmVzIG9mIGFib3V0IDI1MCwgdGhlIG9wdGltaXplciBvZnRlbiBmaW5kcyBhIHNvbHV0aW9uIGF0IGFyb3VuZCA0MDAwLiBUaGFua2Z1bGx5LCB0aGUgc29mdG1heCB0ZW1wZXJhdHVyZSBpcyBub3QgYSBwYXJhbWV0ZXIgd2UncmUgZGVlcGx5IGludGVyZXN0ZWQgaW4gaW50ZXJwcmV0aW5nLCBzbyBhcyBsb25nIGFzIGl0IGRvZXNuJ3QgaW50ZXJmZXJlIHdpdGggcmVjb3Zlcnkgb2YgZ2FtbWEgKHdoaWNoIGl0IGFwcGVhcnMgbm90IHRvKSwgaXQgc3VmZmljZXMgdGhhdCBsYXJnZSAoaW52ZXJzZSkgdGVtcGVyYXR1cmVzIHJlZmxlY3QgInN0cm9uZyB3ZWlnaHRpbmcuIgoKYGBge3IgcmVjb3Zlcnktc3Itc29mdG1heC10ZW1wZXJhdHVyZX0KcmVjb3Zlcnlfc3IgJT4lCiAgZ2dwbG90KGFlcyh4PXRydWVfc29mdG1heF90ZW1wZXJhdHVyZSwgeT1lc3Rfc29mdG1heF90ZW1wZXJhdHVyZSkpICsKICB0aGVtZV9jdXN0b20oKSArCiAgZ2VvbV9hYmxpbmUoc2xvcGUgPSAxLCBpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJibHVlIikgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAwLjI1KSArCiAgeGxhYigiVHJ1ZSBzb2Z0bWF4IHRlbXBlcmF0dXJlIikgKwogIHlsYWIoIkVzdGltYXRlZCBzb2Z0bWF4IHRlbXBlcmF0dXJlIikgKwogIGdndGl0bGUoIlBhcmFtZXRlciByZWNvdmVyeTogU3VjY2Vzc29yIFJlcC4iKQoKcmVjb3Zlcnlfc3IgJT4lCiAgZ2dwbG90KGFlcyh4PXRydWVfc29mdG1heF90ZW1wZXJhdHVyZSwgeT1lc3Rfc29mdG1heF90ZW1wZXJhdHVyZSkpICsKICB0aGVtZV9jdXN0b20oKSArCiAgZ2VvbV9hYmxpbmUoc2xvcGUgPSAxLCBpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJibHVlIikgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAwLjI1KSArCiAgeGxhYigiVHJ1ZSBzb2Z0bWF4IHRlbXBlcmF0dXJlIikgKwogIHlsYWIoIkVzdGltYXRlZCBzb2Z0bWF4IHRlbXBlcmF0dXJlIikgKwogIGdndGl0bGUoIlBhcmFtZXRlciByZWNvdmVyeTogU3VjY2Vzc29yIFJlcC4iKSArCiAgY29vcmRfY2FydGVzaWFuKHlsaW0gPSBjKC0xMDAsIDIwMDAwKSkKCnBsb3RfcmVjb3Zlcnlfc3Jfc29mdG1heCA8LSByZWNvdmVyeV9zciAlPiUKICBnZ3Bsb3QoYWVzKHg9dHJ1ZV9zb2Z0bWF4X3RlbXBlcmF0dXJlLCB5PWVzdF9zb2Z0bWF4X3RlbXBlcmF0dXJlKSkgKwogIHRoZW1lX2N1c3RvbSgpICsKICBnZW9tX2FibGluZShzbG9wZSA9IDEsIGludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRhc2hlZCIsIGNvbG9yID0gImJsdWUiKSArCiAgZ2VvbV9wb2ludChhbHBoYSA9IDAuMjUpICsKICB4bGFiKCJUcnVlIHNvZnRtYXggdGVtcGVyYXR1cmUiKSArCiAgeWxhYigiRXN0aW1hdGVkIHNvZnRtYXggdGVtcGVyYXR1cmUiKSArCiAgZ2d0aXRsZSgiUGFyYW1ldGVyIHJlY292ZXJ5OiBTdWNjZXNzb3IgUmVwLiIpICsKICBjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoLTEwMCwgNTAwMCkpCgpwbG90X3JlY292ZXJ5X3NyX3NvZnRtYXgKCnJlY292ZXJ5X3NyICU+JQogIHdpdGgoCiAgICBjb3IudGVzdCgKICAgICAgdHJ1ZV9zb2Z0bWF4X3RlbXBlcmF0dXJlLCBlc3Rfc29mdG1heF90ZW1wZXJhdHVyZSwKICAgICAgbWV0aG9kID0gInNwZWFybWFuIgogICAgKQogICkgJT4lCiAgdGlkeSgpICU+JQogIGthYmxlX2N1c3RvbSgiUGFyYW1ldGVyIHJlY292ZXJ5OiBTdWNjZXNzb3IgUmVwLiIpCmBgYAoKIyMgUGxvdCBmb3Igc3VwcGxlbWVudAoKV2UnbGwgbm93IGNvbWJpbmUgYWxsIG9mIHRoZXNlIHBhcmFtZXRlciByZWNvdmVyeSBwbG90cy4uLgoKYGBge3IgcGxvdC1yZWNvdmVyeS1jb21iaW5lZH0KI3wgZmlnLndpZHRoPTgsIGZpZy5oZWlnaHQ9NgoKcGxvdF9yZWNvdmVyeV9jb21iaW5lZCA8LSAoCiAgKHBsb3RfcmVjb3ZlcnlfYmZzX2JhY2t3YXJkICsgZ2d0aXRsZSgiQkZTLWJhY2t3YXJkIikpICsKICAgIChwbG90X3JlY292ZXJ5X2Jmc19mb3J3YXJkICsgZ2d0aXRsZSgiQkZTLWZvcndhcmQiKSkgKwogICAgKHBsb3RfcmVjb3ZlcnlfaWRlYWxfb2JzICsgZ2d0aXRsZSgiSWRlYWwgb2JzZXJ2ZXIiKSkKKSAvCiAgKAogICAgKHBsb3RfcmVjb3Zlcnlfc3JfZ2FtbWEgKyBnZ3RpdGxlKCJTdWNjZXNzb3IgUmVwcmVzZW50YXRpb24gZ2FtbWEiKSkgKwogICAgICAocGxvdF9yZWNvdmVyeV9zcl9zb2Z0bWF4ICsgZ2d0aXRsZSgiU3VjY2Vzc29yIFJlcHJlc2VudGF0aW9uIHNvZnRtYXgiKSkKICApICsKICBwbG90X2Fubm90YXRpb24oCiAgICB0aXRsZSA9ICJQYXJhbWV0ZXIgcmVjb3ZlcnkiLAogICAgdGhlbWUgPSB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSksCiAgICB0YWdfbGV2ZWxzID0gIkEiLAogICAgdGFnX3N1ZmZpeCA9ICIuIgogICkKCnBsb3RfcmVjb3ZlcnlfY29tYmluZWQKCmlmIChrbml0dGluZykgewogIGdnc2F2ZSgKICAgIGZpbGVuYW1lID0gaGVyZSgib3V0cHV0cyIsIHdvcmtmbG93X25hbWUsICJwYXJhbV9yZWNvdmVyeV9jb21iaW5lZC5wZGYiKSwKICAgIHBsb3QgPSBwbG90X3JlY292ZXJ5X2NvbWJpbmVkLAogICAgd2lkdGggPSA4LCBoZWlnaHQgPSA2LAogICAgdW5pdHMgPSAiaW4iLCBkcGkgPSAzMDAKICApCiAgCiAgIyBSZWR1bmRhbnQgY29weQogIGdnc2F2ZSgKICAgIGZpbGVuYW1lID0gaGVyZSgiZmlndXJlcyIsICJzdXBwX3BhcmFtX3JlY292ZXJ5LnBkZiIpLAogICAgcGxvdCA9IHBsb3RfcmVjb3ZlcnlfY29tYmluZWQsCiAgICB3aWR0aCA9IDgsIGhlaWdodCA9IDYsCiAgICB1bml0cyA9ICJpbiIsIGRwaSA9IDMwMAogICkKfQpgYGAKCg==